package com.at.bigdata.spark.core.wc

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 *
 * @author cdhuangchao3
 * @date 2023/3/6 8:40 PM
 */
object Spark01_WordCount {

  def main(args: Array[String]): Unit = {
    // App
    // Spark框架
    // TODO 建立和spark框架的连接
    val sparConf = new SparkConf()
      .setMaster("local")
      .setAppName("WordCount")
    val sc = new SparkContext(sparConf)
    // TODO 执行业务操作
    // 1.读取一行
    val lines: RDD[String] = sc.textFile("datas")
    // 2.拆分成1个1个单词
    //   扁平化
    val words = lines.flatMap(_.split(" "))
    // 3.将数据根据单词分组，进行统计
    val wordGroup: RDD[(String, Iterable[String])] = words.groupBy(word => word)
    // 4.对分组后数据，进行转换
    val word2Count = wordGroup.map {
      case (word, list) => {
        println("111111")
        (word, list.size)
      }
    }
    // 5.将转换结果采集到控制台打印
    val array = word2Count.collect()
    array.foreach(println)
    // TODO 关闭连接
    sc.stop()
  }
}
